{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import log_loss\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapath = 'E:/csdn/week3/data/'\n",
    "train = pd.read_csv(datapath+\"RentListingInquries_FE_train.csv\")\n",
    "test = pd.read_csv(datapath+\"RentListingInquries_FE_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train,test = train_test_split(train,train_size = 0.2,random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "x_train = train.drop(['interest_level'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9870, 227)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, x_train, y_train, cv_folds=5, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    xgtrain = xgb.DMatrix(x_train, label = y_train)   \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(x_train, y_train, eval_metric='mlogloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1,x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 6,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': None,\n",
       " 'n_estimators': 87,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.5}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb1.get_xgb_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8HNW1wPHf2VWzqptcZVvuxgZswBVsbIrBEGJCx2BCCTgkOHQIkHx4eSS0kFAChBLaIxBMCcXBlFBsqjHuvXe5ykWWrK7d8/64I3kRkiXZWq2kPd/PZz/embk7czS73rP33pl7RVUxxhhjAHyRDsAYY0zjYUnBGGNMBUsKxhhjKlhSMMYYU8GSgjHGmAqWFIwxxlSwpGBMCBG5S0Sei3QcxkSKJYUmRkSSRWSDiFwSsi5FRDaJyPkh6waLyPsisldEckRkmYjcKyKtvO1XiEhARPZ7j3Ui8qswxz5GRLLCeYy6qCoeVb1PVa8O0/E2iMip4dh3ODTU+9XUzktzZ0mhiVHV/cAk4DERSfdW/xmYo6pvAYjI8cAM4Bugn6q2BMYBZcDAkN3NVNVkVU0Gzgf+LCLHNMxfYupCRGIiHYOJEqpqjyb4AF4CXgPGALuBjiHbvgYer+H1VwBfV1r3PXBJyPJ4YCmQg0syR4RsO8Jbl+OVGR+y7UxgGZAHbAFuBZKAQiAI7Pcenar5u54EpnmvnwX0rMX56Ad8AuwBVgIXHko8wB+AV7zXZQIKXAlsBvYC1wJDgEXe3/5EyHF6Ap9778cu4FWgpbftn96xCr1j3V6Lc7wB+K13rGIgxlve4v0tK4FTqjgXw4HtgD9k3TnAIu/5UGAOkAvsAB6u5pyOAbKq2ZYGvAxkAxuB3wM+b5sf+Kt3DtYDk73zGFPNvjYAp1az7Rpgjfe+Ti3/zAACPALsBPZ55+jI6t7vSP9/bUqPiAdgj0N846AVsM37j3dlyPokIACMqeH1VxCSFLwvuhygj7fcB8gHxgKxwO3ef844b3kNcJe3fLL3H7Cv99ptwKiQOI/1nlf7JRMSx0veF8BQ70vwVWBKDa9Jwn1pX+m95ljvvAyoazxUnRSeBhKA04Ai4F2gHdDZ+1Ia7ZXv5Z2veCAd+BJ4NGTfP/jyO9g5Dim/AOgCtAD6en9np5D4qkyYwFpgbMjym8Ad3vOZwGXe82RgeDX7qPb9wiWE94AUL45VwC+8bdfivpQzvPP9KYeQFLzP1S7v/YwHHge+9LadDswFWuISxBF4P4yqe7/tUbuHNR81Uaq6F/cLMxF4O2RTK1yz4PbyFSLyZ69fIV9Efh9Sdri3fj+ulvBPYLW37SJgmqp+oqqlwF9wX0zH436JJgMPqGqJqn4OvA9M8F5bCvQXkVRV3auq8+r4572tqt+rahkuKQyqofxZwAZVfVFVy7zj/RvXJFYf8fxRVYtU9b+4L/HXVHWnqm4BvgKOAVDVNd75KlbVbOBhYPRB9nuwc1zub6q6WVULcck+3vtbYlV1g6qurWbfr+G9HyKSgvv1/FrI+eglIm1Vdb+qfleXkyEifi/2O1U1T1U34GoGl3lFLgQeU9Us73P6QF32H+JS4AVVnaeqxcCdwAgRyfT+hhRcDVFUdbmqbgv5+w7n/Y5qlhSaKBGZiPuF9inwYMimvbhmio7lK1T1dnX9Cu/gfkmX+05VW6rrU+gADADu87Z1wjULlO8jiPuV2tnbttlbV26jtw3gPNyX0EYR+UJERtTxz9se8rwAl4AOphswzEtwOSKSg/tC6VBP8ewIeV5YxXIygIi0E5EpIrJFRHKBV4C2B9nvwc5xuc0h29cAN+JqMzu9Y3WqZt//As4VkXjgXGCeqpYf6xe4WsoKEZktImcdJMaqtMXVEDeGrAt9/zuFxl3peV1UPj/7cU1znb0fIk/gmhp3iMizIpLqFT3c9zuqWVJogkSkHa499Rrgl8CFInIigKrm49rhz63LPlV1B+7X9U+9VVtxX7blxxRcM8YWb1sXEQn9/HT1tqGqs1X1bFwTy7vAG+WHqUtMdbAZ+MJLcOWPZFX9VQPHc7+3z6NVNRWYiGvaKFf5eAc7x1W+RlX/paojvdcpP/xBEFpuGe4L9QzgElySKN+2WlUn4M7Hg8BbIpJU+z+TXbhf491C1lW8/7jmm4yQbV3qsO9Qlc9PEtCGA5+zv6nqcbgfM32A27z11b3fphYsKTRNTwDvqup0r8p8O/AP71ch3vJVInKHl0AQkQyge3U7FJE2uM7Ipd6qN4CfiMgpIhIL3ILr7PwWl3TygdtFJFZExuCSyRQRiRORS0UkzWsSycU1e4D7hd1GRNLq6TyUex/oIyKXefHEisgQETmigeNJwXUi54hIZ7wvqRA7gB4hywc7xz8iIn1F5GTvfS7C1VICVZX1/Au4HjgR16dQvp+JIpLu1UxyvNXV7kdEEkIfuJroG8C93uXQ3YCbcTWj8r/rBhHpLCItcZ3jNYmtdJwYL/4rRWSQ9zffB8xS1Q3e+zvMO2/53vkI1PB+m9qIdKeGPer2AH6G+wXVstL6z4B7Q5aHAR/g/tPnAEuAe4E23vYrcP9Zyq+82Ylrc24Xso9zcB2G+4Av8DpuvW0DvHX7vDLneOvjgI9wzVi5wGxgZMjrXsA1AeRQ/dVHfwpZHkMNndNeub64K5ayvf1/juuLqFM8VN3RHBNSPouQTnzcF+HvQ87JXO98LsB9yWeFlD0b2OQd69ZanOMN/LBj+mhc308erjP+/arOYUj5rrgv8GmV1r/ivd/7cT8CflbN68d4f3/lRy9c39Ur3vneDNzNgauPYnA12d24q49uwtUspJrjbKjiGH/ytl2L6zQv/3szvPWn4K442s+BK72Sa3q/7VHzQ7wTbIwxYSEiZwBPq2q3GgubiLPmI2NMvRKRFiJypojEeM1o/4O7yME0AVZTME2CiIwCPqxqm7qrp0wjISKJuKawfrh+j2nADaqaG9HATK1YUjDGGFPBmo+MMcZUaHKDbLVt21YzMzMjHYYxxjQpc+fO3aWq6TWVa3JJITMzkzlz5kQ6DGOMaVJEZGPNpaz5yBhjTAhLCsYYYypYUjDGGFPBkoIxxpgKlhSMMcZUsKRgjDGmgiUFY4wxFaImKWzbV8iHi7fVXNAYY6JY1CSFC5+eyeRXZ7MnvyTSoRhjTKMVNUnh1aPmsyB+EvPW7ai5sDHGRKmoSQodMrqTIoVsXv59pEMxxphGK2qSQly3YQCUbbKkYIwx1YmapEBaZ/Ji02mXu5jCEpvH2xhjqhI9SQEobDeIgaxhYVZOpEMxxphGKaqSQkqvEWT6drBk9bpIh2KMMY1SVCWFFt1dv8K+Nd9FOBJjjGmcoiop0OkYgvhI2jmfQNDmpjbGmMqiKynEJZGb2pv+wVWs3J4X6WiMMabRCVtSEJEXRGSniCypZruIyN9EZI2ILBKRY8MVS6iYrkMY5FvLnA27GuJwxhjTpISzpvASMO4g288AenuPScBTYYylQlKP4aRKAZtWLWqIwxljTJMStqSgql8Cew5S5GzgZXW+A1qKSMdwxVNOMoYAENw8B1XrVzDGmFCR7FPoDGwOWc7y1v2IiEwSkTkiMic7O/vwjtq2DyUxyfQoXsaWnMLD25cxxjQzkUwKUsW6Kn+6q+qzqjpYVQenp6cf3lF9PkraD/L6FfYe3r6MMaaZiWRSyAK6hCxnAFsb4sCJ3YfTTzbxl2kLGuJwxhjTZEQyKUwFfu5dhTQc2KeqDTILjq/LEGIkyNG+9davYIwxIWLCtWMReQ0YA7QVkSzgf4BYAFV9GvgAOBNYAxQAV4Yrlh/JGOz+yV/Cxt0FZLZNarBDG2NMYxa2pKCqE2rYrsB14Tr+QSW1pTQtk8F7VvHVml2WFIwxxhNddzSHiOkxiuH+lXy90mZiM8aYclGbFCRzFKnsZ/e6+ZQGgpEOxxhjGoWoTQpkngDA0WWLWbjZ5lcwxhiI5qSQlkGgZXeG+5bx1WobB8kYYyCakwLg7z6SETEr+XqV9SsYYwxEeVIgcyQpup/CLYvZV1ga6WiMMSbiojspdHP9CsNkGTPX7o5wMMYYE3nRnRRadkFbZTIyZgVfrT7MgfaMMaYZiO6kAEjmSIb7V/D1qp2RDsUYYyIu6pMCmaNICuaRmLOSddn7Ix2NMcZElCUFr19huG8Z/11mVyEZY6KbJYWWXaBVJqclruHjpdsjHY0xxkSUJQWAzJEco0tZsGkPO3KLIh2NMcZEjCUFgMxRJJTl0l82WhOSMSaqWVIA6HkyAOemLOPjJdaEZIyJXpYUAJLbQcdBnBa3mO/W7WZfgd3dbIyJTpYUyvUeS0b+EpKCeXy2wpqQjDHRyZJCuV5jEQ1yVtIKuwrJGBO1LCmUyxgMCS05L3U5X6zKprAkEOmIjDGmwVlSKOfzQ8+TObJwNsWlZXxpYyEZY6KQJYVQvU8jrmgXR/k3cdfbiyMdjTHGNDhLCqF6nQLAdRnrKSwNkF9cFuGAjDGmYYU1KYjIOBFZKSJrROSOKrZ3E5HPRGSRiMwQkYxwxlMj79LU44PzKCgJWIezMSbqhC0piIgfeBI4A+gPTBCR/pWK/QV4WVWPBu4B7g9XPLXWeyzJu+bTv1UZ78zfEulojDGmQYWzpjAUWKOq61S1BJgCnF2pTH/gM+/59Cq2Nzzv0tRfd9nM12t2sX2fjYVkjIke4UwKnYHNIctZ3rpQC4HzvOfnACki0qbyjkRkkojMEZE52dlhvirIuzR1tMxHFd5bYLUFY0z0CGdSkCrWaaXlW4HRIjIfGA1sAX7Uu6uqz6rqYFUdnJ6eXv+RhvL5oc84UjZ+yuAuybw9bwuqlcM2xpjmKZxJIQvoErKcAWwNLaCqW1X1XFU9Bvidt25fGGOqnf7joSiHa7ttZeWOPJZty410RMYY0yDCmRRmA71FpLuIxAEXA1NDC4hIWxEpj+FO4IUwxlN7PU+GuGRGlXxDrF94e541IRljokPYkoKqlgGTgY+B5cAbqrpURO4RkfFesTHAShFZBbQH7g1XPHUS2wJ6n0b8mg85pW8b3luwlbJAMNJRGWNM2MWEc+eq+gHwQaV1d4c8fwt4K5wxHLL+42Hp2yRtn82u/d35fMVOThvQIdJRGWNMWNkdzdXpNRZiWvBQ//V0Skvg/2ZuiHRExhgTdpYUqhOfDL1OwbfifSYO78I3a3azakdepKMyxpiwsqRwMP1/Bvu3M7HzTuJifPzftxsiHZExxoSVJYWD6XM6+ONIXf8hZw/sxNvztthUncaYZs2SwsEkpEKPk2DZVC4f0Y3C0gBvzt1c8+uMMaaJsqRQkz3rYN8mjmQtQzJb8fLMjQSCdoezMaZ5sqRQk2s+A388LJrC5cdnsmlPAdNX7Ix0VMYYExaWFGqSkAb9fgKL3+L0fq2J9Qs3v7Eg0lEZY0xYWFKojYEXQ+EeYtd9xm2n9yW3qIy5G/dGOipjjKl3lhRqo+fJkJQOC6dw6bButEqM5YnPV0c6KmOMqXeWFGrDHwtHXQCrPiIpkMvVo3owfWU2S7ZEfkBXY4ypT5YUamvgxRAogaXvcNmIbqQmxPC41RaMMc2MJYXa6nA0tOsPC6eQmhDLFSd05+OlO1i53Ya+MMY0H5YUaksEjr4Isr6H3Wu56oRMkuL8PDF9TaQjM8aYemNJoS6OvhAQWPgaLRPjuGxEJv9ZuJXxj38d6ciMMaZeWFKoi9RO7r6Fbx6DQBnXjOpOcnwM7dMSIh2ZMcbUC0sKdXX2k67DeeUHtEmO51djevLJsh18t253pCMzxpjDZkmhrvqMg7QuMPsfAPxiZHc6pSXwp2nLCNqYSMaYJs6SQl35Y2DwlbD+S8heSUKsn9vG9WXJllzeXbAl0tEZY8xhsaRwKI69HPxxMPs5AM4e2JmjM9J46OOVFJYEIhycMcYcOksKhyKpLQw4Bxa8BsV5+HzC7848gm37injuq3WRjs4YYw5ZWJOCiIwTkZUiskZE7qhie1cRmS4i80VkkYicGc546tWQa6AkDxa9DsCwHm1olRjLI5+uYtPugggHZ4wxhyZsSUFE/MCTwBlAf2CCiPSvVOz3wBuqegxwMfD3cMVT7zIGQ8eB8P1zoK6D+YMbRpEYF8Nd7yxG1TqdjTFNTzhrCkOBNaq6TlVLgCnA2ZXKKJDqPU8DtoYxnvolAkMnQfZy1+kMdExrwW/P6MfXa3bx5tysCAdojDF1F86k0BkIndA4y1sX6g/ARBHJAj4AfhPGeOrfkedDcnv4+pGKVZcO7crQzNb86f1l7MwrimBwxhhTd+FMClLFusptKhOAl1Q1AzgT+KeI/CgmEZkkInNEZE52dnYYQj1EsQkw4jpYNx22zAPA5xPuP+8oisqC/GHq0ggHaIwxdRPOpJAFdAlZzuDHzUO/AN4AUNWZQALQtvKOVPVZVR2sqoPT09PDFO4hGnyVG/ri64crVvVMT+aGU3rzweLtjH34iwgGZ4wxdRPOpDAb6C0i3UUkDteRPLVSmU3AKQAicgQuKTSiqkAtxKfAsGth+X9g54qK1ZNO7MFRndPYtb+YnbnWjGSMaRrClhRUtQyYDHwMLMddZbRURO4RkfFesVuAa0RkIfAacIU2xct2hl0LsYnwzaMVq2L9Ph65aBAFJQFu//ciuxrJGNMk1JgURKSniMR7z8eIyPUi0rI2O1fVD1S1j6r2VNV7vXV3q+pU7/kyVT1BVQeq6iBV/e/h/DERk9gajrsSFr0BezdWrO7VLpm7zjyCGSuzeWXWpggGaIwxtVObmsK/gYCI9AKeB7oD/wprVE3R8ZNBfG5Y7RA/H9GNE/ukc++0ZazN3h+h4IwxpnZqkxSCXlPQOcCjqnoT0DG8YTVBqZ3gmIkw7+Uf1BZEhIfOP5qygPLTx7+mqNTGRjLGNF61SQqlIjIBuBx431sXG76QmrATb3O1hRkP/GB1+9QEnp54HIWlAX5r/QvGmEasNknhSmAEcK+qrheR7sAr4Q2riUrrDMMmwcLXYOfyH2w6tX97bj2tL+8t2MpTX6yNUIDGGHNwNSYFrzP4elV9TURaASmq+kBNr4taI292l6l+/qcfbfr1mJ6MH9iJhz5eyafLdkQgOGOMObjaXH00Q0RSRaQ1sBB4UUQerul1USuxNRx/Pax4HzbP/sEmEeHP5x/NUZ3TuGHKfFZsz41QkMYYU7XaNB+lqWoucC7woqoeB5wa3rCauOG/gqR0+Ox/K0ZQLZcQ6+fZywZTEgjy08e/ZmtOYYSCNMaYH6tNUogRkY7AhRzoaDYHE5/sOp03fAVrPv3R5g5pCUydPJKEGD+Xv/A9OQUlEQjSGGN+rDZJ4R7cXclrVXW2iPQAVoc3rGbguCugdU/48LdQVvyjzUd0TOWZnx/Hxt0FXPPyHLtU1RjTKNSmo/lNVT1aVX/lLa9T1fPCH1oTFxMPZ/wZ9qyFmU9WWeT4nm15+KKBzN6wlxH3f0ZZINjAQRpjzA/VpqM5Q0TeEZGdIrJDRP4tIhkNEVyT1/tU6HcWfPkQ7Kt60p2zju5Et9aJ7C0o5aY3FlpiMMZEVG2aj17EjW7aCTdJzn+8daY2Tr8PNAgf/67aIl/cfhJ3ntGP/yzcyq1vLiQQtJvbjDGRUZukkK6qL6pqmfd4CWhkkxo0Yq26wahbYNm7sHZ6tcV+Obont53el3cXbOW2tywxGGMiozZJYZeITBQRv/eYCOwOd2DNyvHXQ6vu8MFtUFr93ArXndSLm8f24e15Wxh676eUlFlTkjGmYdUmKVyFuxx1O7ANOB839IWprdgEOOth2L0aZtx30KLXn9KbLq1asDu/hCtf+p7cotIGCtIYY2p39dEmVR2vqumq2k5Vf4a7kc3URc+T4djL4dvHf3Snc2Vf/fZk/nrBQGat28OFT89kh83cZoxpIIc689rN9RpFtDjtT5DSCd779UGbkQDOOy6DF64YwqodeYx6cDrzNu1toCCNMdHsUJOC1GsU0SIhFcb/DXatqrEZCeDEPun075iKCFz49Eye+2qdDbttjAmrQ00K9s10qHqdEtKM9H2Nxd+/fhTf33UqJ/drx5+mLeeal+fasBjGmLCpNimISJ6I5FbxyMPds2AO1Wl/gtQM+PfVULSvxuJpibE8c9lx3H1Wfz5bvoMh937K16t3NUCgxphoU21SUNUUVU2t4pGiqjENGWSzk5AK5z3n7nKedsuPRlKtiohw1cjuvDf5BLq0TmTi87P43/8stTGTjDH16lCbj8zh6joMxtwBi9+EhVNq/bKjM1oy7TejuOL4TF78ZgOD7vkvczfuCWOgxphoEtakICLjRGSliKwRkTuq2P6IiCzwHqtEJCec8TQ6o26BbifAB7fC7tpP0dkizs8fxg+gb/sUgkE4/+mZ/O6dxewrtHsajDGHR8J1NYuI+IFVwFggC5gNTFDVZdWU/w1wjKpedbD9Dh48WOfMmVPf4UbOvix46gRolQlXfexudKuD/OIy/vrfVbz07Xr8PqFr60Q+vXk0InaBmDHmABGZq6qDayoXzprCUGCNN9R2CTAFOPsg5ScAr4UxnsYpLQN+9nfYtgA+vK3OL0+Kj+Hun/bnvetGEuv3sTY7n/OfnsmCzdFV6TLG1I/aDJ1d1VVIm73htHsc5KWdgc0hy1neuqqO0Q3oDnxel+CbjX4/cU1J816GOYc2AO1RGWks/sPpPHjeUWzcXcDPnvyGm15fwOY9BfUcrDGmOatNTeFh4DbcF3oGcCvwD9wv/xcO8rqq2i+qa6u6GHhLVau8lEZEJonIHBGZk52dXYuQm6CTfgc9T3GD5tUwDEZ1/D7hoiFdmXHbGDqlJfDugi2c9JcZ3PXOYpsL2hhTKzX2KYjILFUdVmndd6o6XEQWqurAal43AviDqp7uLd8JoKr3V1F2PnCdqn5bU8DNrk8hVMEeeHYMBEph0gxIaX9Yu9u+r4gnp69hyuxNlAWUdinxvDd5JB3S6tZvYYxp+uqzTyEoIheKiM97XBiy7WAZZTbQW0S6i0gcrjYwtYpA+wKtgJm1iKV5S2wNF78K+7fD48dByeE1/XRIS+CPPzuSGbedRNuUeHbmFXPin6dz93tL2LbPag7GmB+rTVK4FLgM2Ok9LgMmikgLYHJ1L1LVMm/7x8By4A1VXSoi94jI+JCiE4ApaoP6OB2Oggv/CSX74e1rIHj4N6d1btmC2b87lS9vP4nzjuvMv2Zt4vj7P+e4P37Ct2t32XhKxpgKYbskNVyadfNRqO+ego/ugOG/hnE/anE7LJv3FPD81+t5e14WuUVlJMT4aJsSz6tXD6Nbm6R6PZYxpnGobfNRbfoUMoDHgRNwzUVfAzeoatUz0YdZ1CQFgA/vgFlPwbgHYPiv6n33RaUBPli8jf95byl5xWUAHNU5jZ8c3ZHT+renR3pyvR/TGBMZ9ZkUPgH+BfzTWzURuFRVxx52lIcgqpJCMAAP9YbC3XDe83DU+WE7VNbeAj5cvJ33F29joXePQ2abRE7u156T+7VjSPdWxMf4w3Z8Y0x41WdSWKCqg2pa11CiKikAlBbCK+fDppmuE7rvGWE/ZNbeAqav2MlnK3byxapsVKFFrJ/je7ZhdN90RvdJt2YmY5qY+kwKnwIvceBu4wnAlap6yuEGeSiiLikAFOXCy2fDjqVw6ZvQY3SDHbqgpIzv1u3mi5XZzFiVzcbd7oqozDaJFJYESGsRy5vXHk9aYmyDxWSMqbv6TApdgSeAEbg+hW+B61V1U30EWldRmRTA3cPw4pmQswl+/i50GRqRMDbsyufL1dl8sTKb6St3ElQQgX4dUtmTX0xKfAzPXzGErq0TbfwlYxqReksK1ez8RlV99JAiO0xRmxQA8rbDYwPdzW2XT4XMkRENp7gswPxNOcxat4dZ63fz3brdBL2Pk98nJMX5ufz4TAZmtGRgl5akp8RHNF5jolm4k8ImVe16SJEdpqhOCgC521xTUs5GuOhV6H1qpCOqUBoIsnJ7Houy9vHIp6vILy6juCxIwMsU8TE+Th/QgYVZOaTEx/DOdScQ67cpPYxpCOFOCptVtcshRXaYoj4pAOTvgn+eAzuXw/kvQP/xNb8mQgpLAizduo+b31jA/uIAsX5hR24x4JLEgE6pbM0pJDEuhkcvHkTP9GSS4m1iP2Pqm9UUmrvCHHj0SCjOg5/+DY67PNIR1Yqqcs7fv2F/URlj+rZjYVYOczfurWh2AoiL8TGiRxt6tUvm8+U7aBHn55Wrh9M6KS5ygRvTxB12UhCRPKoe20iAFpGap9mSQoji/fDm5bDmUxh9h5veswl27pYFgmzcU8DqHfv507RlFJYEaJ+awNrs/RSXBSvKtUqMpTSgJMT6uGRYN95fuJX4GB/PXDaY9mnxdh+FMQcR1ppCJFlSqCRQCv+5ARa8CsdcBmc9Cv7m0fwSCCrn/v0bikoDXDC4C2uz83l/0VaKS4OUBoNU/ujG+oU4v48TerVlYVYOcX4fN5zah3Yp8aSnxNMxLYG0FrF2VZSJSpYUookqTL8PvvwzJLSCGxdCQlqkowqrkrIgW3IK2bK3kK37CtmWU8Qr322kJBCkfWo8a7PzKzq4Q7WI9aMocX4fJ/drR7vUBD5aso0Yn4/fn3UELRPjaJsUT4e0BOJirBPcNB+WFKLR3Jdg2i3Qphdc8rqb9zmKFZSUsTO3mOz9xdz59mJKyoKM7d+ed+ZlURJQWibGsjOvmJKQJqpyPoEOqQnkFZcR6/dx5lEd+Hz5TmL8wm9O7k2rxDge/mQlMT4fL101lLQWsfh9VgMxjZclhWi17gt44+fg87tLVruNiHREjZqqkltUxt78EvYWlHDHvxdRGlDOGtiJrD0FfLJsB6XBIElxMezOLznovvw+oWNaAsnxMWzZW4jPJwzr3pqEWD8z1+7G74MLBnfh3flb8PuEm8b2ITk+hr98vBK/T3j8kmO5+fUF+HzCG78cYUnG1CtLCtFs1xp4eiSUFcEZD8LQSU2yA7qxUVXyisvIyS9lb0EJt7+1kLKgMnF4N577ah2lAWVkr7bkFZcxa91uAqp0aZVIUWmALTmFBIMQRH/UF1KdOL+PgCo+gfapCezaX4xPhN7tU0iI8bGYuta5AAAXHklEQVRiex4+gVOOaM9Xq7Pxi3Dp8G60iPXzz5kb8PmE28f145FPVuETeOiCgSTE+mlR/ojzEx/jsz6WKGFJIdoV7oV3roVVH0G/s+DsJ6BFq0hHFfVUlQuenkkgqPz1woHsLy7j9rcWEQgq153Ui0c/XUVQlfOP60JBSYB35mcRVBjVuy0zVmYTVOXojJYUlwZYsnUfwaC7KmtHXnGVfSi14RNIjo8h1u8jt6gUnwg90pNJivOzakceIsIxXVuyYJMbPXdI99b4BL5fvwcRYVTvtnyzZheC8NOBnUiK9/Pu/C2ICNeM6k6M30es31dxIUCs30dinJ+k+BiSE2JoEeuuGguqS5gxfiEhxk9CrEtaPqsx1QtLCsZ1QH/3d/jkfyClg7vRLUJjJpnwuuiZmagq/7x6GIUlAa548XuCQbjv3KO4/a2FBILKTWP7UlwWcIknCBcN7cKr320kEIRxR3agNBDkv0u3E1Q4tlsrCkrKWLA5BxR6tktm7c79KNCtTRKqysbdBShK+9QEtu8rIqhKfIyf/JKyWteGakME/CIEVBEgJSGW/OIyRKBdSgJ+n7Aztwi8GlVQ1d0gqZCeEk92nrtZsn1aPD4RduQWIQi92iWzflc+Im4eEZ8Ii7fsQ1UZ0CmNJVv2oUCvdsmoKqt37q8ou3ybq6WN7pNOfIyfGSt3gsAZR3bkwyXbQOGU/u0JBJTPVuxAFU7o1ZagKt+u3Y3gXvvV6mwQYdyADsT6fe61wPhBnZi6YCsA5x7bGb8I/563hZaJsUy7ftQhnkdLCqbclrnw4hlQVgwn/x5G3uz6HIwJA1WlsDRAaZlSEghSFgxSWqaUBoOUlLlHQUmA/OIy7v9wOUFVrh3dE0F4+ou1qCo/Pz6TF75eT1Ddl2IgqLwzfwuqyukDOvDhku2oKif2aUcgGOSr1btQXI3KL8KXq7MBGNO3nfvCBk7snU5A1ZVVZVCXlszduBcF+nVIIRBUVu7IQxD6d0xl+fZcBBjYpSU+ERZuziGoSt8OKSzbmktQoV1qPMWlQXbtL0aBpDg/+SUBBGiZGEeMT9hTUIIAnVq2QAS25hS6hJUaz459xShKcnwMpQElv8RNdhXn91ESCIKCzycVtcDMNonMuO2kQ3pfLCmYHyraB/+5EZa+DZmj4NxnIbVTpKMyxtSSes1rh9qcVtukYBdiR4uENNd8dPaTrubw6FHwxLBIR2WMqSURaZD+FUsK0UQEjpkIv/wS2h8Ju1bA25PcOErGGIMlhejUtjdc7Y2XtPgt+PsIN36SMSbqhTUpiMg4EVkpImtE5I5qylwoIstEZKmI/Cuc8ZgQ/lg46U6XHIr2wivnuUtY83dHOjJjTASFLSmIiB94EjgD6A9MEJH+lcr0Bu4ETlDVAcCN4YrHVKPzsXD7BjjxNlj8Jjw5BBa+Tr1eU2iMaTLCWVMYCqxR1XWqWgJMAc6uVOYa4ElV3QugqjvDGI+pTmyCu1T1l19Cq+7wziR4oCts/j7SkRljGlg4k0JnYHPIcpa3LlQfoI+IfCMi34nIuKp2JCKTRGSOiMzJzs4OU7iG9gPgF/91w2/HtoDnx8LrE92wGcaYqBDOpFDVtVOV2yRigN7AGGAC8JyItPzRi1SfVdXBqjo4PT293gM1IXx+GHwlXD8fTvodrPgAnjjOjb663ypyxjR34UwKWUDoPM4ZwNYqyrynqqWquh5YiUsSJtLikmD07XDLChhyjRuW+2/HwIwHoSQ/0tEZY8IknElhNtBbRLqLSBxwMTC1Upl3gZMARKQtrjlpXRhjMnWV3A5+8hf49SzoeTLMuA8ezIR5L0MwEOnojDH1LGxJQVXLgMnAx8By4A1VXSoi94jIeK/Yx8BuEVkGTAduU1W7JrIxatsLLvon/OIT6DgIpv4Gnh4Fqz+xK5WMaUZs7CNTd6qw7F13X0NZEXQe7C5p7XO6zdtgTCNlYx+Z8BGBAefAHZvgrEcgfye8dhE8MwqWvA2BskhHaIw5RJYUzKGLiYfBV8Fv5sHZf4ddq+GtK12H9My/Q3FepCM0xtSRJQVz+PyxcMylcNdWNy90Wmf4+E54oBt8cjfkVr7ozBjTWFlSMPXH54cjzoKrPoIOA6FFS/j2cTdM99u/hG2LIh2hMaYGMZEOwDRT137p/t27Ab57Cr5/FhZNcRP8jJgMvU8Dn/0mMaaxsf+VJrxaZcIZD8Lt62DsPbBnneuUvq8TzHkBSgsjHaExJoQlBdMwWrSCE26AGxZC276uqen9m+CRI91d0vm7Ih2hMQa7T8FEiips/Mb1Oaz6CBA4+iIYeg10Ps7udzCmntX2PgXrUzCRIQKZI90jeyV8/w9Y+Jrrd4hLdk1NR57nOquNMQ3Gagqm8SjOg2dGQ942KC2AmAQ44qcwcAJ0Hw1++w1jzKGymoJpeuJT4Pp5rmlp2wKY/wrMfdHNCJfYxiWIAee62oXPH+lojWmWrKZgGrfSIljzKSx9B1Z+CKX54I9zl7UOugTa2kjrxtRGbWsKlhRM01FSAM+OcWMtFeWCBiAuBU79H1eDSGoT6QiNabQsKZjmLW8HvHCamw2utAB8MRCfCknpMGm6myTIGFPB+hRM85bS3t3zALB9ibtqadYzULgHHuoNsYmQ1BYmzYDYhEhGakyTYjUF03wEA7DxW1jyluukDpa55qW+42D7YkhoBb/4KNJRGhMR1nxkoltZCWz4Epa+Cyveh8K9ID7oe6a7L6JFa7j6v5GO0pgGY0nBmHKBUnhmDBTuBgTyvKG8u58IR4yHha+7uSGunBbJKI0JK+tTMKacPxZ+/Y17rupukCvYDbnb4INb3fq4ZJh+vxu9tdMxNoKriVpWUzDRLXslvHqB66Au3g8o+GJh4MWuFtFjtKtFGNPEWfORMXWVvxteHAcFe6CsGEryQPxu/KWxf4TeYyG5XaSjNOaQWPORMXWV1AYmz3bPy4ph3RcwdbLrpH7v1259XDIMvhK6nQBdh7shwY1pRsJaUxCRccBjgB94TlUfqLT9CuAhYIu36glVfe5g+7SagmlwqrB9Ebx+mUsQZUUQKHHb4pLhhBuh7xnQfoAN+W0arYg3H4mIH1gFjAWygNnABFVdFlLmCmCwqk6u7X4tKZiIKy2ELXPhnV+5JFGS59b74+GYia6ZqfuJdle1aVQaQ/PRUGCNqq7zApoCnA0sO+irjGnsYlu4kVpvWuyW87bDiz9xndULX4M5zwPiht0YNgm6DIcuQyAhLaJhG1Mb4UwKnYHNIctZwLAqyp0nIifiahU3qermygVEZBIwCaBr165hCNWYw5DSAa6f656XFbu7qt/9NRTvg68edgP3gWtqGvZL6HESdBlqVzWZRimczUcXAKer6tXe8mXAUFX9TUiZNsB+VS0WkWuBC1X15IPt15qPTJNSvB+2zIH3JkPRPijJd0lCfC459BjtJhDqcJTNEWHCqjE0H2UBXUKWM4CtoQVUdXfI4j+AB8MYjzENLz4ZeoyBm5a45aJceP40KMqB3C3wyd1uvfih16nuvomEVLjmc3fTnTENLJxJYTbQW0S6464uuhi4JLSAiHRU1W3e4nhgeRjjMSbyElLhuu8OLOdth5d+CsW5sHc95Gxw6x/o5pqY9qxzzU6X/8fmizANImxJQVXLRGQy8DHuktQXVHWpiNwDzFHVqcD1IjIeKAP2AFeEKx5jGqWUDvCb2QeWnxvrahM9ToQN30DORrf+oR7u6qa4JDjhBugyzA3HYcOCm3pmdzQb05gV7HH3SGxbBN/8zV3+WlbkbRQ3r/XwX8GKae5qp6s+jGi4pvGK+H0K4WJJwUS9/F2w+Xs3mF/RPjfznAYBcXdZdxkGqz52l8D+4uNIR2saicbQ0WyMCYekttDvTPcAKMyBF8a5BBEohZlPQrDUbXv+dMjd6objuPpTiImLXNymSbCkYExT16LlDzuvSwu9vokcNxzHvk3ucX8GdBwI+7Jcs9Mlr0OrTBuaw/yAJQVjmpvYFvCrrw8sP+ddAtt7rBueY/92N9HQ3waBL8YliCHXQKdB0HEQpHayRBHFLCkY09xVnnY0UAY7l7mb6qY/4Dqvv/qL1y+Bm0+i92mQMRiW/NvNc21zW0cN62g2xkBJAexYAm//0iWJhDTYvcbbKG6o8JxN7gqnK6e5JivTpFhHszGm9uIS3c1yN8w/sK5gjxvor3ifSxT7Nrn1D3aDlt2gZD/EJsEZD0KHIyGtizU7NQOWFIwxVUtsDdfNPLD83OkuORx1PmxbCKs+cnNdT5ngtvv8rkbRcZC7JDYuGa75zBJFE2NJwRhTO1dXcc9DcR7sXA7vXOsG+yvOg1lPH5iE6IFu7t/4ZFejyBji7uI2jZb1KRhj6ldZCTw/1o0Q2+NEWPSGSxh43zX+eJckRkx2Q3V0GmTTmjYAu6PZGNN4lBW7oTqyZsNXf3X9ERXDdQBtekHnwZD1veunmPgWJLe3pqd6ZEnBGNO4FexxI8SW5EH7I13CyN95YLv4IKYFDDgHOh8DnY5182Db5ESHxK4+MsY0bomt4dffHFhWdXdb71oJe9a7WetKC12H9oJXDpRr28d1cMcmwun3uSufWmaCz9fgf0JzZDUFY0zjpgr7NsMrF0BpvhuqY82nP2x+Er8bVnzghAN3ZrftbRMVhbDmI2NM81ZS4K582rEEZtzvOrODAZc4ABA3zWnedpcwfvqYa6aK0smKLCkYY6JPMODuxJ4y0SWH9L6w/qsDo8YC+ONcZ/aQq1zSaH+UGxjQ37xb061PwRgTfXx+lwhCZ7MD2L8Tdix1tYpvHnO1iq8fBQ247f44aN3TdXTHJcNZj7jLZRNbN/zfEGFWUzDGRKfSInj+NDdJUb8zIXsVrJv+w76KmHjoPtp1bq/8yDVDNdF5KaymYIwxBxObANd++eP1hTmwbQFMvd7VKHK3wfovDySL+zNcUohLgZPuhHYDoF0/lzCaAaspGGNMTYJBN3FRyX7ofSrMe9k9Lx9uHCAmAXqe7G7Si0uE855zN+U1kmRhHc3GGBNOwSDsXe/mpvjoLtcMldjG3WcRyh/vJj4adIlrhpr9vLvHovI8F2FmScEYYyLhhTPcTXcjb4Rdq2HWM1BW4O63KC04UK5FawiWuYQx6hZo1x8++6O7t+LKafUeVqNICiIyDngM8APPqeoD1ZQ7H3gTGKKqB/3Gt6RgjGmSgkHIzYJ/XeSSRo/RsORtlyiCZQfK+WPdEOTZq1yNYvzfXDNUcrvDGgsq4klBRPzAKmAskAXMBiao6rJK5VKAaUAcMNmSgjEmqqi6G+x2LoNpN7ub8tI6uzkrQvssxA+te8BvDu37rzFcfTQUWKOq67yApgBnA8sqlfsj8Gfg1jDGYowxjZMIpHZ0jxsWHlj/wpkQKIaT7oLda+Hrh90AgWEWzqTQGdgcspwFDAstICLHAF1U9X0RqTYpiMgkYBJA165dwxCqMcY0Mld9cOB5r1Ng2KQGOWw4hxWsqvGroq1KRHzAI8AtNe1IVZ9V1cGqOjg9Pb0eQzTGGBMqnEkhC+gSspwBbA1ZTgGOBGaIyAZgODBVRGps8zLGGBMe4UwKs4HeItJdROKAi4Gp5RtVdZ+qtlXVTFXNBL4DxtfU0WyMMSZ8wpYUVLUMmAx8DCwH3lDVpSJyj4iMD9dxjTHGHLqwjn2kqh8AH1Rad3c1ZceEMxZjjDE1s/nrjDHGVLCkYIwxpoIlBWOMMRUsKRhjjKnQ5EZJFZFsYOMhvrwtsKsew2lO7NxUz85N9ezcVK0xnpduqlrj3b9NLikcDhGZU5sBoaKRnZvq2bmpnp2bqjXl82LNR8YYYypYUjDGGFMh2pLCs5EOoBGzc1M9OzfVs3NTtSZ7XqKqT8EYY8zBRVtNwRhjzEFYUjDGGFMhapKCiIwTkZUiskZE7oh0PJEkIl1EZLqILBeRpSJyg7e+tYh8IiKrvX9bRTrWSBARv4jMF5H3veXuIjLLOy+ve0PBRx0RaSkib4nICu+zM8I+M46I3OT9X1oiIq+JSEJT/dxERVIQET/wJHAG0B+YICL9IxtVRJUBt6jqEbjJja7zzscdwGeq2hv4zFuORjfghnsv9yDwiHde9gK/iEhUkfcY8JGq9gMG4s5R1H9mRKQzcD0wWFWPBPy4+WOa5OcmKpICMBRYo6rrVLUEmAKcHeGYIkZVt6nqPO95Hu4/d2fcOfk/r9j/AT+LTISRIyIZwE+A57xlAU4G3vKKROt5SQVOBJ4HUNUSVc3BPjPlYoAWIhIDJALbaKKfm2hJCp2BzSHLWd66qCcimcAxwCygvapuA5c4gHaRiyxiHgVuB4Lechsgx5s0CqL3s9MDyAZe9JrWnhORJOwzg6puAf4CbMIlg33AXJro5yZakoJUsS7qr8UVkWTg38CNqpob6XgiTUTOAnaq6tzQ1VUUjcbPTgxwLPCUqh4D5BOFTUVV8fpRzga6A52AJFxTdWVN4nMTLUkhC+gSspwBbI1QLI2CiMTiEsKrqvq2t3qHiHT0tncEdkYqvgg5ARgvIhtwTYwn42oOLb1mAYjez04WkKWqs7zlt3BJIto/MwCnAutVNVtVS4G3geNpop+baEkKs4He3tUAcbhOoKkRjilivHby54HlqvpwyKapwOXe88uB9xo6tkhS1TtVNUNVM3Gfkc9V9VJgOnC+VyzqzguAqm4HNotIX2/VKcAyovwz49kEDBeRRO//Vvm5aZKfm6i5o1lEzsT96vMDL6jqvREOKWJEZCTwFbCYA23nd+H6Fd4AuuI+6Beo6p6IBBlhIjIGuFVVzxKRHriaQ2tgPjBRVYsjGV8kiMggXAd8HLAOuBL3wzLqPzMi8r/ARbgr++YDV+P6EJrc5yZqkoIxxpiaRUvzkTHGmFqwpGCMMaaCJQVjjDEVLCkYY4ypYEnBGGNMBUsKxhhjKlhSMKYWRGSQd69L+fL4+hqCXURuFJHE+tiXMYfL7lMwphZE5Arc0MiTw7DvDd6+d9XhNX5VDdR3LMZYTcE0KyKS6U0A8w9v0pP/ikiLasr2FJGPRGSuiHwlIv289Rd4k6UsFJEvvaFR7gEuEpEFInKRiFwhIk945V8Skae8iYvWichoEXnBi+OlkOM9JSJzvLj+11t3PW4QtekiMt1bN0FEFnsxPBjy+v0ico+IzAJGiMgDIrJMRBaJyF/Cc0ZN1FFVe9ij2TyATNxQA4O85TdwwwtUVfYzoLf3fBhurCNww3909p639P69Angi5LUVy8BLuOEMBDdaZi5wFO5H19yQWFp7//qBGcDR3vIGoK33vBNuuIh03MiknwM/87YpcGH5voCVHKjtt4z0ubdH83hYTcE0R+tVdYH3fC4uUfyAN2z48cCbIrIAeAbo6G3+BnhJRK7BfYHXxn9UVXEJZYeqLlbVILA05PgXisg83Dg4A3CzAFY2BJihbsTNMuBV3OQ2AAHcyLbgEk8R8JyInAsU1DJOYw4qpuYixjQ5oYOOBYCqmo98uElQBlXeoKrXisgw3AxsC7yB4Gp7zGCl4weBGBHpDtwKDFHVvV6zUkIV+6lq/oZyRer1I6hqmYgMxY3IeTEwGTfUtzGHxWoKJiqpm1RovYhcAG44cREZ6D3vqaqzVPVuYBduLo48IOUwDpmKm5hmn4i054eTsITuexYwWkTaenOLTwC+qLwzr6aTpqofADcCtUlcxtTIagomml0KPCUivwdicf0CC4GHRKQ37lf7Z966TcAdXlPT/XU9kKouFJH5uOakdbgmqnLPAh+KyDZVPUlE7sSNxS/AB6pa1Tj8KcB7IpLglbuprjEZUxW7JNUYY0wFaz4yxhhTwZqPTLMnIk/i5l8O9ZiqvhiJeIxpzKz5yBhjTAVrPjLGGFPBkoIxxpgKlhSMMcZUsKRgjDGmwv8DwAmemZiCk70AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一步确定弱学习器的数目为87个，接下来第二步调整树的参数：max_depth min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [3, 4, 5, 6, 7, 8, 9]}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = list(range(3,10,1))\n",
    "param_test2_1 = dict(max_depth = max_depth)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.63538, std: 0.00976, params: {'max_depth': 3},\n",
       "  mean: -0.62683, std: 0.01043, params: {'max_depth': 4},\n",
       "  mean: -0.62620, std: 0.01221, params: {'max_depth': 5},\n",
       "  mean: -0.62675, std: 0.01051, params: {'max_depth': 6},\n",
       "  mean: -0.63230, std: 0.00837, params: {'max_depth': 7},\n",
       "  mean: -0.63428, std: 0.01098, params: {'max_depth': 8},\n",
       "  mean: -0.64483, std: 0.00929, params: {'max_depth': 9}],\n",
       " {'max_depth': 5},\n",
       " -0.6262025930355902)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch2_1.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_weight': [1, 2, 3, 4, 5]}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_weight = list(range(1,6,1))\n",
    "param_test2_2 = dict(min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62620, std: 0.01221, params: {'min_child_weight': 1},\n",
       "  mean: -0.62595, std: 0.01251, params: {'min_child_weight': 2},\n",
       "  mean: -0.62648, std: 0.01147, params: {'min_child_weight': 3},\n",
       "  mean: -0.62585, std: 0.01050, params: {'min_child_weight': 4},\n",
       "  mean: -0.62486, std: 0.01098, params: {'min_child_weight': 5}],\n",
       " {'min_child_weight': 5},\n",
       " -0.6248642766120227)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch2_2.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62603, std: 0.01088, params: {'min_child_weight': 6},\n",
       "  mean: -0.62695, std: 0.01139, params: {'min_child_weight': 7},\n",
       "  mean: -0.62677, std: 0.01065, params: {'min_child_weight': 8},\n",
       "  mean: -0.62668, std: 0.01191, params: {'min_child_weight': 9}],\n",
       " {'min_child_weight': 6},\n",
       " -0.6260252577011127)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_weight = list(range(6,10,1))\n",
    "param_test2_3 = dict(min_child_weight=min_child_weight)\n",
    "param_test2_3\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch2_3 = GridSearchCV(xgb2_3, param_grid = param_test2_3, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch2_3.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_3.grid_scores_, gsearch2_3.best_params_,     gsearch2_3.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "max_depth = 5 min_child_weight = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62486, std: 0.01098, params: {'gamma': 0.0},\n",
       "  mean: -0.62512, std: 0.01141, params: {'gamma': 0.1},\n",
       "  mean: -0.62413, std: 0.01168, params: {'gamma': 0.2},\n",
       "  mean: -0.62511, std: 0.01112, params: {'gamma': 0.3},\n",
       "  mean: -0.62554, std: 0.01127, params: {'gamma': 0.4}],\n",
       " {'gamma': 0.2},\n",
       " -0.6241338679542918)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test3 = { 'gamma':[i/10.0 for i in range(0,5)]}\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch3 = GridSearchCV(xgb3, param_grid = param_test3, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch3.fit(x_train , y_train)\n",
    "\n",
    "gsearch3.grid_scores_, gsearch3.best_params_,     gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调整树的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "param_test4_1 = dict(subsample=subsample)\n",
    "param_test4_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.63217, std: 0.01137, params: {'subsample': 0.3},\n",
       "  mean: -0.62842, std: 0.01025, params: {'subsample': 0.4},\n",
       "  mean: -0.62413, std: 0.01168, params: {'subsample': 0.5},\n",
       "  mean: -0.62185, std: 0.00929, params: {'subsample': 0.6},\n",
       "  mean: -0.62269, std: 0.01094, params: {'subsample': 0.7},\n",
       "  mean: -0.62354, std: 0.01032, params: {'subsample': 0.8}],\n",
       " {'subsample': 0.6},\n",
       " -0.6218521517105099)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0.2,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch4_1 = GridSearchCV(xgb4_1, param_grid = param_test4_1, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch4_1.fit(x_train , y_train)\n",
    "\n",
    "gsearch4_1.grid_scores_, gsearch4_1.best_params_,     gsearch4_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62328, std: 0.00922, params: {'colsample_bytree': 0.75},\n",
       "  mean: -0.62185, std: 0.00929, params: {'colsample_bytree': 0.8},\n",
       "  mean: -0.62465, std: 0.01079, params: {'colsample_bytree': 0.85}],\n",
       " {'colsample_bytree': 0.8},\n",
       " -0.6218521517105099)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4_2 = {\n",
    "    'colsample_bytree':[i/100.0 for i in range(75,90,5)]\n",
    "}\n",
    "\n",
    "xgb4_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0.2,\n",
    "        subsample=0.6,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch4_2 = GridSearchCV(xgb4_2, param_grid = param_test4_2, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch4_2.fit(x_train , y_train)\n",
    "\n",
    "gsearch4_2.grid_scores_, gsearch4_2.best_params_,     gsearch4_2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对正则化进行交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62185, std: 0.00929, params: {'reg_alpha': 1e-05},\n",
       "  mean: -0.62250, std: 0.01027, params: {'reg_alpha': 0.01},\n",
       "  mean: -0.62384, std: 0.00977, params: {'reg_alpha': 0.1},\n",
       "  mean: -0.62347, std: 0.01005, params: {'reg_alpha': 1},\n",
       "  mean: -0.70087, std: 0.00155, params: {'reg_alpha': 100}],\n",
       " {'reg_alpha': 1e-05},\n",
       " -0.6218521577722215)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test5 = {'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100]}\n",
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0.2,\n",
    "        subsample=0.6,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch5.fit(x_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.62185, std: 0.00929, params: {'reg_alpha': 0},\n",
       "  mean: -0.62145, std: 0.00973, params: {'reg_alpha': 0.001},\n",
       "  mean: -0.62163, std: 0.01041, params: {'reg_alpha': 0.005},\n",
       "  mean: -0.62250, std: 0.01027, params: {'reg_alpha': 0.01},\n",
       "  mean: -0.62230, std: 0.00970, params: {'reg_alpha': 0.05}],\n",
       " {'reg_alpha': 0.001},\n",
       " -0.6214495348193039)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test6 = {'reg_alpha':[0, 0.001, 0.005, 0.01, 0.05]}\n",
    "xgb6 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=87,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0.2,\n",
    "        subsample=0.6,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch6 = GridSearchCV(xgb6, param_grid = param_test6, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch6.fit(x_train , y_train)\n",
    "\n",
    "gsearch6.grid_scores_, gsearch6.best_params_,     gsearch6.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(xgb6,open(\"xgb_model.pkl\",'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0.2, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=5, min_child_weight=5, missing=nan,\n",
       "       n_estimators=87, n_jobs=1, nthread=None, objective='multi:softprob',\n",
       "       random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "       seed=3, silent=True, subsample=0.6)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb = pickle.load(open(\"xgb_model.pkl\",'rb'))\n",
    "xgb.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapath = 'E:/csdn/week3/data/'\n",
    "x_test = pd.read_csv(datapath+\"RentListingInquries_FE_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = xgb.predict_proba(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
